# -*- coding: utf-8 -*-
import os
import logging
from  scrapy import *;
from xuekubao.redisoper import CRedis
from xuekubao.items import XuekubaoItem
from xuekubao.items import Shiji21PaperItem
from scrapy_redis.spiders import RedisCrawlSpider
import  urlparse
import urllib
import json
from xuekubao.genidoper import *;
from xuekubao.items import Shiji21Item

class Shiji21Spider(RedisCrawlSpider):
    name='shiji21'
    redis_key = 'fshiji21_spider:start_urls'
    rules=()

    def initlog(self):
        logger = logging.getLogger("simple_example")
        logger.setLevel(logging.DEBUG)
        # 建立一个filehandler来把日志记录在文件里，级别为debug以上
        fh = logging.FileHandler("/logs/shijian21.log")
        fh.setLevel(logging.DEBUG)
        eh = logging.FileHandler("/logs/shijian21RROR.log")
        eh.setLevel(logging.ERROR)
        # 建立一个streamhandler来把日志打在CMD窗口上，级别为error以上
        ch = logging.StreamHandler()
        ch.setLevel(logging.ERROR)
        # 设置日志格式
        formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
        ch.setFormatter(formatter)
        fh.setFormatter(formatter)
        eh.setFormatter(formatter)
        # 将相应的handler添加在logger对象中
        logger.addHandler(ch)
        logger.addHandler(fh)
        logger.addHandler(eh)
        return logger;

    def __init__(self):
        self.redis_key = 'fshiji21_spider:start_urls';
        self.r = CRedis();
        self.base_url='http://tiku.21cnjy.com';
        self.slogger = self.initlog();
        self.initStartUrls();
        self.count=1;

        super(Shiji21Spider, self).__init__()


    def url2Dict(self,url):
        query = urlparse.urlparse(url).query
        return dict([(k, v[0]) for k, v in urlparse.parse_qs(query).items()])

    def initStartUrls(self):
        self.r.remove(self.redis_key)
        if (self.r.exits(self.redis_key)):
            return;

        #适用年级(21:初一，22：初二，23:初三)
        grades=('21','22','23','31','32','33')
        #试卷年份(0，全部)
        paperyears=('2010','2011','2012','2013','2014','2015','2016')
        #2语文,4英语,6物理,8历史 9思想政治,10地理,11生物
        channels=('2','3','4','6','7','8','9','10','11')
        #试卷类型
        #(0:全部，1:单元试卷，2：专题试卷，3约考试卷，4开学考试，5其中考试，6期末考试7中考模拟，8中考真倦，11 水平会考)
        papertypes=('1','2','3','4','5','6','7','8','11','12','14')
        #试卷省份0:全部，1北京，2上海，3天津，4重庆，5江苏，6广东，7，浙江，8山东，9湖北，10吉林，11河北，12福建
        #13安徽 14四川，15湖南，16江西，17辽宁，18黑龙江 19山西，20河南，21山西，22云南，23甘肃，24贵州，25广西，26海南
        #27青海，28内蒙古，29西藏，30宁夏，31新疆
        paperareas=('1','2','3','4','5','6','7','8','9','10','11','12','13','14','15'
                    '16','17','18','19','20','21','22')
        base_url='http://tiku.21cnjy.com/tiku.php';
        for grade in grades:
            for paperyear in paperyears:
                for papertype in papertypes:
                    for paperarea in paperareas:
                        for channel in channels:
                         url='http://tiku.21cnjy.com/tiku.php?mod=%s&channel=%s&xd=%s&d=%s&t=%s&y=%s&g=%s'\
                             %('paper',channel,'2',paperarea,papertype,paperyear,grade)
                         self.r.lpush(self.redis_key,url)

        return;

        # 列表页
   #解析试卷列表页
    def parse_start_url(self, response):
        try:
            temp = response.url;
            self.slogger.debug('start url=%s' % (temp))
            params=self.url2Dict(temp)
            extra = {'channel': params['channel'], 'grade':params['g'],'paperyear':params['y'],'papertype':params['t'],'paperarea':params['d']}
            shijuans = response.xpath("//div[@class='frame lists_data']//ul[@class='list_one']/li/a[1]/@href").extract();
            if len(shijuans) == 0:
                return;
            for u in shijuans:
                u = self.base_url+"/" + u;
                yield Request(u, callback=self.parse_shijuan, meta=extra, errback=self.parse_shijuan);
            # nextpage
            nexturl = response.xpath("//div[@class='pg']/strong/following-sibling::a[1]/@href").extract();
            if (len(nexturl) == 0):
                return;
            nexturl = self.base_url + nexturl[0];
            yield Request(nexturl, callback=self.parse_start_url, errback=self.parse_start_url);
        except Exception, e:
            self.logger.error(e)

    #解析单个试卷的试题列表页
    def parse_shijuan(self,response):
        try:
            extra=response.meta;
            pid=getUUid()
            extra['blpaper']=pid;
            papername=response.xpath("//div[@class='shijuan']/b/text()").extract()[0];
            item = Shiji21PaperItem();
            item['id'] = pid
            item['channel'] = extra['channel']
            item['grade'] = extra['grade']
            item['paperyear'] = extra['paperyear']
            item['papertype'] = extra['papertype']
            item['paperarea'] = extra['paperarea']
            item['papername'] = papername;
            yield item
            tis=response.xpath("//p[@class='btns']/a[1]/@href").extract();
            if(len(tis)==0):
                return;
            for u in tis:
                u=self.base_url+u;
                yield Request(u, callback=self.parse_ti_deta, meta=extra, errback=self.parse_ti_deta);
        except Exception, e:
                self.logger.error(e)




    #解析单个实体页
    def parse_ti_deta(self,response):
        try:
            extra=response.meta;
            question=response.xpath("//div[@class='answer_detail']/dl/dt[1]").extract()[0];
            answer=response.xpath("//span[@class='option']/following-sibling::i[1]").extract()[0];
            jiexi=response.xpath("//span[@class='parsing']/following-sibling::i[1]").extract()[0];
            item = Shiji21Item();
            item['id']=getUUid()
            item['channel']=extra['channel']
            item['grade']=extra['grade']
            item['paperyear']=extra['paperyear']
            item['papertype']=extra['papertype']
            item['paperarea']=extra['paperarea']
            item['tquestion']=question
            item['tanswer']=answer
            item['tjiexi']=jiexi;
            item['blpaper']=extra['blpaper']
            #self.count=self.count+1;

            return item;
        except Exception, e:
            self.logger.error(e)
















